﻿using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using System.Text;
using System.Threading.Tasks;

namespace HTTPAgilitySpider
{
    public class HttpHelper
    {
        public static string DownloadUrl(string url)
        {
            return DownloadHtml(url, Encoding.UTF8);
        }

        public static string DownloadHtml(string url, Encoding encode)
        {
            string html = string.Empty;
            try
            {
                // 创建一个发送http请求对象
                HttpWebRequest request = HttpWebRequest.Create(url) as HttpWebRequest;
                request.Timeout = 3000; // 设置请求过期时间
                request.UserAgent = "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.106 Safari/537.36";
                //request.UserAgent = "User - Agent:Mozilla / 5.0(iPhone; CPU iPhone OS 7_1_2 like Mac OS X) App leWebKit/ 537.51.2(KHTML, like Gecko) Version / 7.0 Mobile / 11D257 Safari / 9537.53";
                request.ContentType = "text/html; charset=utf-8";// "text/html;charset=gbk";// 

                //request.Host = "";
                //request.Headers.Add("Cookie", @"_ga=GA1.2.610783074.1541838820; pgv_pvi=3725304832");
                //request.Headers.Add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8");
                //request.Headers.Add("Accept-Encoding", "gzip, deflate, sdch");
                //request.Headers.Add("Referer", @"");

                //Encoding enc = Encoding.GetEncoding("GB2312"); // 如果是乱码就改成 utf-8 / GB2312

                // 创建一个存储缓存的对象容器
                request.CookieContainer = new CookieContainer();



                // 获取请求返回对象
                using (HttpWebResponse response = request.GetResponse() as HttpWebResponse)
                {

                    if (response.StatusCode != HttpStatusCode.OK) // 说明请求失败了
                    {
                        Console.WriteLine(string.Format("抓取{0}地址返回失败,response.StatusCode为{1}", url, response.StatusCode));
                        //logger.Warn(string.Format("抓取{0}地址返回失败,response.StatusCode为{1}", url, response.StatusCode));
                    }
                    else
                    {
                        try
                        {
                            StreamReader sr = new StreamReader(response.GetResponseStream(), encode);
                            html = sr.ReadToEnd();
                            sr.Close();
                        }
                        catch (Exception ex)
                        {
                            Console.WriteLine(string.Format($"DownloadHtml抓取{url}失败"), ex);
                            //logger.Error(string.Format($"DownloadHtml抓取{url}失败"), ex);
                            html = null;
                        }

                    }
                }



            }
            catch (Exception ex)
            {

            }
            return html;
        }
    }
}
